from scrapy.spider import BaseSpider
from scrapy.http import Request, Response
from scrapy.selector import HtmlXPathSelector

from courseScraper.items import CoursescraperItem

DEBUG = 0
DEBUG_DEPARTMENTS = {
        'Psychology': [107, 117, 122, 133, 143, 160, 164, '166AC', '167AC'],
        'Rhetoric': ['103A', 105, 110, 110, 170, 174, 175, 177],
        }

DEPARTMENTS = {
        'Anthropology': [149, '160AC', 161, 166],
        'Cognitive+Science': [1, 'c100', 'c101', 'c102', 'c103', 'c104', 'c108', 'c110', 'c124', 'c126', 'c127', 'c131', 'c140', 'c142'],
        'Computer+Science': [160, 170, 186, 188, 287, 288],
        'Economics': [119],
        'Education': ['140AC', '224A', '229A'],
        'Information': [146],
        'Integrative+Biology': [245],
        'Linguistics': [100, 106, 110, 115, 120, 121, 123, 130, 'c139', 150, 151, 158, 170, 181],
        'Molecular+and+Cell+Biology': ['c160', 163, '160L', 164, 165, 166],
        'Music': ['c108'],
        'Native+American+Studies': [151],
        'Philosophy': [107, 111, 121, 122, 128, 130, 132, 133, 135, 136, 138, '140A', 153, 174, 176, 178, 185, 186, 188],
        'Political+Science': [161, '164A'],
        'Psychology': [107, 110, 111, 114, 117, 122, 133, 143, 160, 164, '166AC', '167AC'],
        'Rhetoric': ['103A', 105, 110, 110, 170, 174, 175, 177],
        'Sociology': [150, '150A'],
        'Visual+Science': [265],
        }

SEMESTER = 'FL'

class NewCourseSpider(BaseSpider):
    name = "new-course-spider"
    allowed_domains = ["berkeley.edu"]
    start_urls = []

    def __init__(self, category=None):
        to_search = DEBUG_DEPARTMENTS if DEBUG else DEPARTMENTS
        for d in to_search:
            for num in to_search[d]:
                new_url = 'http://osoc.berkeley.edu/OSOC/osoc?p_term=%s&p_deptname=%s&p_course=%s' % (SEMESTER, d, num)
                self.start_urls.append(new_url)

    def parse(self, response):
        hxs = HtmlXPathSelector(response)
        check = hxs.select('//table/tr/td/font').extract()
        MISSING_STRING = u'No classes match your request.'
        for c in check:

            if MISSING_STRING not in c:
                new_item = CoursescraperItem()

                req = response.request.url
                req = req.replace('http://osoc.berkeley.edu/OSOC/osoc?','')
                for i in req.split('&'):
                    kv = i.split('=')
                    if kv[0] == 'p_deptname':
                        new_item['department'] = kv[1]
                    elif kv[0] == 'p_course':
                        new_item['course_num'] = kv[1]
                return new_item

